import requests
from bs4 import BeautifulSoup


def get_novel_chapters():
    data = []
    url = 'https://www.bodekang.org/43_43265/'

    r = requests.get(url)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    for dd in soup.find_all('dd'):
        link = dd.find('a')
        if not link:
            continue
        else:
            data.append(['https://www.bodekang.org' + link['href'], link.text.strip()])
    return data


def get_chapter_content(url):
    r = requests.get(url)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    return soup.find('div', id='content').get_text().strip().split('\n')[0].replace('　　', '\n    ')


f = open('novel/元尊.txt', 'a', encoding='utf-8')
for chapter in get_novel_chapters():
    print(f'开始爬取 {chapter[1]}')
    f.write(f'{chapter[1]}\n\n')  # 写入目录
    f.write('    ' + get_chapter_content(chapter[0]) + '\n\n')  # 写入内容
f.close()

print('爬取完成')
